{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-29T03:06:25.482096Z",
     "start_time": "2024-02-29T03:06:18.831463Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-29T03:06:25.499028Z",
     "start_time": "2024-02-29T03:06:25.486375Z"
    }
   },
   "outputs": [],
   "source": [
    "# Renaming\n",
    "# tic_list = [tic[:-4] for tic in os.listdir(\"./CSI\")]\n",
    "# for i in range(len(tic_list)):\n",
    "#     data = pd.read_csv(f\"./CSI/{tic_list[i]}.csv\", index_col=0)\n",
    "#     os.remove(f\"./CSI/{tic_list[i]}.csv\")\n",
    "#     if tic_list[i][-2:] == \"SS\":\n",
    "#         mark = \"SH\"\n",
    "#     else:\n",
    "#         mark = \"SZ\"\n",
    "#     tic_list[i] = mark + tic_list[i][:6]\n",
    "#     data.to_csv(f\"./CSI/{tic_list[i]}.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-29T03:19:50.479892Z",
     "start_time": "2024-02-29T03:19:50.455270Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[12664:MainThread](2024-02-29 11:19:50,460) INFO - qlib.Initialization - [config.py:416] - default_conf: client.\n",
      "[12664:MainThread](2024-02-29 11:19:50,464) WARNING - qlib.Initialization - [__init__.py:64] - auto_path is False, please make sure None is mounted\n",
      "[12664:MainThread](2024-02-29 11:19:50,467) INFO - qlib.Initialization - [__init__.py:74] - qlib successfully initialized based on client settings.\n",
      "[12664:MainThread](2024-02-29 11:19:50,469) INFO - qlib.Initialization - [__init__.py:76] - data_path={'__DEFAULT_FREQ': WindowsPath('C:/Users/胡逸凡/.qlib/qlib_data/cn_data')}\n"
     ]
    }
   ],
   "source": [
    "import qlib\n",
    "from qlib.config import REG_CN\n",
    "from qlib.contrib.data.handler import Alpha158\n",
    "\n",
    "qlib.init(provider_uri=\"~/.qlib/CSI_data\", region=REG_CN)\n",
    "data_handler_config = {\n",
    "    \"start_time\": \"2018-01-01\",\n",
    "    \"end_time\": \"2022-05-01\",\n",
    "    \"instruments\": \"all\",\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "instrument: {'__DEFAULT_FREQ': 'C:\\\\Users\\\\胡逸凡\\\\.qlib\\\\qlib_data\\\\cn_data'} does not contain data for day",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mTypeError\u001B[0m                                 Traceback (most recent call last)",
      "File \u001B[1;32mE:\\anaconda3\\lib\\site-packages\\pyqlib-0.9.3.99-py3.10-win-amd64.egg\\qlib\\data\\data.py:1187\u001B[0m, in \u001B[0;36mBaseProvider.features\u001B[1;34m(self, instruments, fields, start_time, end_time, freq, disk_cache, inst_processors)\u001B[0m\n\u001B[0;32m   1186\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m-> 1187\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mDatasetD\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdataset\u001B[49m\u001B[43m(\u001B[49m\n\u001B[0;32m   1188\u001B[0m \u001B[43m        \u001B[49m\u001B[43minstruments\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfields\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mstart_time\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mend_time\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfreq\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdisk_cache\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43minst_processors\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43minst_processors\u001B[49m\n\u001B[0;32m   1189\u001B[0m \u001B[43m    \u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m   1190\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mTypeError\u001B[39;00m:\n",
      "\u001B[1;31mTypeError\u001B[0m: LocalDatasetProvider.dataset() got multiple values for argument 'inst_processors'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001B[1;31mValueError\u001B[0m                                Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[10], line 1\u001B[0m\n\u001B[1;32m----> 1\u001B[0m h \u001B[38;5;241m=\u001B[39m Alpha158(\u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mdata_handler_config)\n",
      "File \u001B[1;32mE:\\anaconda3\\lib\\site-packages\\pyqlib-0.9.3.99-py3.10-win-amd64.egg\\qlib\\contrib\\data\\handler.py:173\u001B[0m, in \u001B[0;36mAlpha158.__init__\u001B[1;34m(self, instruments, start_time, end_time, freq, infer_processors, learn_processors, fit_start_time, fit_end_time, process_type, filter_pipe, inst_processors, **kwargs)\u001B[0m\n\u001B[0;32m    159\u001B[0m learn_processors \u001B[38;5;241m=\u001B[39m check_transform_proc(learn_processors, fit_start_time, fit_end_time)\n\u001B[0;32m    161\u001B[0m data_loader \u001B[38;5;241m=\u001B[39m {\n\u001B[0;32m    162\u001B[0m     \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mclass\u001B[39m\u001B[38;5;124m\"\u001B[39m: \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mQlibDataLoader\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[0;32m    163\u001B[0m     \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mkwargs\u001B[39m\u001B[38;5;124m\"\u001B[39m: {\n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m    171\u001B[0m     },\n\u001B[0;32m    172\u001B[0m }\n\u001B[1;32m--> 173\u001B[0m \u001B[38;5;28msuper\u001B[39m()\u001B[38;5;241m.\u001B[39m\u001B[38;5;21m__init__\u001B[39m(\n\u001B[0;32m    174\u001B[0m     instruments\u001B[38;5;241m=\u001B[39minstruments,\n\u001B[0;32m    175\u001B[0m     start_time\u001B[38;5;241m=\u001B[39mstart_time,\n\u001B[0;32m    176\u001B[0m     end_time\u001B[38;5;241m=\u001B[39mend_time,\n\u001B[0;32m    177\u001B[0m     data_loader\u001B[38;5;241m=\u001B[39mdata_loader,\n\u001B[0;32m    178\u001B[0m     infer_processors\u001B[38;5;241m=\u001B[39minfer_processors,\n\u001B[0;32m    179\u001B[0m     learn_processors\u001B[38;5;241m=\u001B[39mlearn_processors,\n\u001B[0;32m    180\u001B[0m     process_type\u001B[38;5;241m=\u001B[39mprocess_type,\n\u001B[0;32m    181\u001B[0m     \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs\n\u001B[0;32m    182\u001B[0m )\n",
      "File \u001B[1;32mE:\\anaconda3\\lib\\site-packages\\pyqlib-0.9.3.99-py3.10-win-amd64.egg\\qlib\\data\\dataset\\handler.py:468\u001B[0m, in \u001B[0;36mDataHandlerLP.__init__\u001B[1;34m(self, instruments, start_time, end_time, data_loader, infer_processors, learn_processors, shared_processors, process_type, drop_raw, **kwargs)\u001B[0m\n\u001B[0;32m    466\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mprocess_type \u001B[38;5;241m=\u001B[39m process_type\n\u001B[0;32m    467\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mdrop_raw \u001B[38;5;241m=\u001B[39m drop_raw\n\u001B[1;32m--> 468\u001B[0m \u001B[38;5;28msuper\u001B[39m()\u001B[38;5;241m.\u001B[39m\u001B[38;5;21m__init__\u001B[39m(instruments, start_time, end_time, data_loader, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n",
      "File \u001B[1;32mE:\\anaconda3\\lib\\site-packages\\pyqlib-0.9.3.99-py3.10-win-amd64.egg\\qlib\\data\\dataset\\handler.py:100\u001B[0m, in \u001B[0;36mDataHandler.__init__\u001B[1;34m(self, instruments, start_time, end_time, data_loader, init_data, fetch_orig)\u001B[0m\n\u001B[0;32m     98\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m init_data:\n\u001B[0;32m     99\u001B[0m     \u001B[38;5;28;01mwith\u001B[39;00m TimeInspector\u001B[38;5;241m.\u001B[39mlogt(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mInit data\u001B[39m\u001B[38;5;124m\"\u001B[39m):\n\u001B[1;32m--> 100\u001B[0m         \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msetup_data\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    101\u001B[0m \u001B[38;5;28msuper\u001B[39m()\u001B[38;5;241m.\u001B[39m\u001B[38;5;21m__init__\u001B[39m()\n",
      "File \u001B[1;32mE:\\anaconda3\\lib\\site-packages\\pyqlib-0.9.3.99-py3.10-win-amd64.egg\\qlib\\data\\dataset\\handler.py:610\u001B[0m, in \u001B[0;36mDataHandlerLP.setup_data\u001B[1;34m(self, init_type, **kwargs)\u001B[0m\n\u001B[0;32m    594\u001B[0m \u001B[38;5;250m\u001B[39m\u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[0;32m    595\u001B[0m \u001B[38;5;124;03mSet up the data in case of running initialization for multiple time\u001B[39;00m\n\u001B[0;32m    596\u001B[0m \n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m    607\u001B[0m \u001B[38;5;124;03m        when we call `init` next time\u001B[39;00m\n\u001B[0;32m    608\u001B[0m \u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[0;32m    609\u001B[0m \u001B[38;5;66;03m# init raw data\u001B[39;00m\n\u001B[1;32m--> 610\u001B[0m \u001B[38;5;28msuper\u001B[39m()\u001B[38;5;241m.\u001B[39msetup_data(\u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n\u001B[0;32m    612\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m TimeInspector\u001B[38;5;241m.\u001B[39mlogt(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mfit & process data\u001B[39m\u001B[38;5;124m\"\u001B[39m):\n\u001B[0;32m    613\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m init_type \u001B[38;5;241m==\u001B[39m DataHandlerLP\u001B[38;5;241m.\u001B[39mIT_FIT_IND:\n",
      "File \u001B[1;32mE:\\anaconda3\\lib\\site-packages\\pyqlib-0.9.3.99-py3.10-win-amd64.egg\\qlib\\data\\dataset\\handler.py:144\u001B[0m, in \u001B[0;36mDataHandler.setup_data\u001B[1;34m(self, enable_cache)\u001B[0m\n\u001B[0;32m    140\u001B[0m \u001B[38;5;66;03m# Setup data.\u001B[39;00m\n\u001B[0;32m    141\u001B[0m \u001B[38;5;66;03m# _data may be with multiple column index level. The outer level indicates the feature set name\u001B[39;00m\n\u001B[0;32m    142\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m TimeInspector\u001B[38;5;241m.\u001B[39mlogt(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mLoading data\u001B[39m\u001B[38;5;124m\"\u001B[39m):\n\u001B[0;32m    143\u001B[0m     \u001B[38;5;66;03m# make sure the fetch method is based on an index-sorted pd.DataFrame\u001B[39;00m\n\u001B[1;32m--> 144\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_data \u001B[38;5;241m=\u001B[39m lazy_sort_index(\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdata_loader\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mload\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43minstruments\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mstart_time\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mend_time\u001B[49m\u001B[43m)\u001B[49m)\n",
      "File \u001B[1;32mE:\\anaconda3\\lib\\site-packages\\pyqlib-0.9.3.99-py3.10-win-amd64.egg\\qlib\\data\\dataset\\loader.py:135\u001B[0m, in \u001B[0;36mDLWParser.load\u001B[1;34m(self, instruments, start_time, end_time)\u001B[0m\n\u001B[0;32m    132\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mload\u001B[39m(\u001B[38;5;28mself\u001B[39m, instruments\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m, start_time\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m, end_time\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m pd\u001B[38;5;241m.\u001B[39mDataFrame:\n\u001B[0;32m    133\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mis_group:\n\u001B[0;32m    134\u001B[0m         df \u001B[38;5;241m=\u001B[39m pd\u001B[38;5;241m.\u001B[39mconcat(\n\u001B[1;32m--> 135\u001B[0m             {\n\u001B[0;32m    136\u001B[0m                 grp: \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mload_group_df(instruments, exprs, names, start_time, end_time, grp)\n\u001B[0;32m    137\u001B[0m                 \u001B[38;5;28;01mfor\u001B[39;00m grp, (exprs, names) \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mfields\u001B[38;5;241m.\u001B[39mitems()\n\u001B[0;32m    138\u001B[0m             },\n\u001B[0;32m    139\u001B[0m             axis\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m1\u001B[39m,\n\u001B[0;32m    140\u001B[0m         )\n\u001B[0;32m    141\u001B[0m     \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m    142\u001B[0m         exprs, names \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mfields\n",
      "File \u001B[1;32mE:\\anaconda3\\lib\\site-packages\\pyqlib-0.9.3.99-py3.10-win-amd64.egg\\qlib\\data\\dataset\\loader.py:136\u001B[0m, in \u001B[0;36m<dictcomp>\u001B[1;34m(.0)\u001B[0m\n\u001B[0;32m    132\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mload\u001B[39m(\u001B[38;5;28mself\u001B[39m, instruments\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m, start_time\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m, end_time\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m pd\u001B[38;5;241m.\u001B[39mDataFrame:\n\u001B[0;32m    133\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mis_group:\n\u001B[0;32m    134\u001B[0m         df \u001B[38;5;241m=\u001B[39m pd\u001B[38;5;241m.\u001B[39mconcat(\n\u001B[0;32m    135\u001B[0m             {\n\u001B[1;32m--> 136\u001B[0m                 grp: \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mload_group_df\u001B[49m\u001B[43m(\u001B[49m\u001B[43minstruments\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mexprs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mnames\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mstart_time\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mend_time\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mgrp\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    137\u001B[0m                 \u001B[38;5;28;01mfor\u001B[39;00m grp, (exprs, names) \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mfields\u001B[38;5;241m.\u001B[39mitems()\n\u001B[0;32m    138\u001B[0m             },\n\u001B[0;32m    139\u001B[0m             axis\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m1\u001B[39m,\n\u001B[0;32m    140\u001B[0m         )\n\u001B[0;32m    141\u001B[0m     \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m    142\u001B[0m         exprs, names \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mfields\n",
      "File \u001B[1;32mE:\\anaconda3\\lib\\site-packages\\pyqlib-0.9.3.99-py3.10-win-amd64.egg\\qlib\\data\\dataset\\loader.py:217\u001B[0m, in \u001B[0;36mQlibDataLoader.load_group_df\u001B[1;34m(self, instruments, exprs, names, start_time, end_time, gp_name)\u001B[0m\n\u001B[0;32m    213\u001B[0m freq \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mfreq[gp_name] \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mfreq, \u001B[38;5;28mdict\u001B[39m) \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mfreq\n\u001B[0;32m    214\u001B[0m inst_processors \u001B[38;5;241m=\u001B[39m (\n\u001B[0;32m    215\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39minst_processors \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39minst_processors, \u001B[38;5;28mlist\u001B[39m) \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39minst_processors\u001B[38;5;241m.\u001B[39mget(gp_name, [])\n\u001B[0;32m    216\u001B[0m )\n\u001B[1;32m--> 217\u001B[0m df \u001B[38;5;241m=\u001B[39m \u001B[43mD\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfeatures\u001B[49m\u001B[43m(\u001B[49m\u001B[43minstruments\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mexprs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mstart_time\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mend_time\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfreq\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mfreq\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43minst_processors\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43minst_processors\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    218\u001B[0m df\u001B[38;5;241m.\u001B[39mcolumns \u001B[38;5;241m=\u001B[39m names\n\u001B[0;32m    219\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mswap_level:\n",
      "File \u001B[1;32mE:\\anaconda3\\lib\\site-packages\\pyqlib-0.9.3.99-py3.10-win-amd64.egg\\qlib\\data\\data.py:1191\u001B[0m, in \u001B[0;36mBaseProvider.features\u001B[1;34m(self, instruments, fields, start_time, end_time, freq, disk_cache, inst_processors)\u001B[0m\n\u001B[0;32m   1187\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m DatasetD\u001B[38;5;241m.\u001B[39mdataset(\n\u001B[0;32m   1188\u001B[0m         instruments, fields, start_time, end_time, freq, disk_cache, inst_processors\u001B[38;5;241m=\u001B[39minst_processors\n\u001B[0;32m   1189\u001B[0m     )\n\u001B[0;32m   1190\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mTypeError\u001B[39;00m:\n\u001B[1;32m-> 1191\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mDatasetD\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdataset\u001B[49m\u001B[43m(\u001B[49m\u001B[43minstruments\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfields\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mstart_time\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mend_time\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfreq\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43minst_processors\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43minst_processors\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[1;32mE:\\anaconda3\\lib\\site-packages\\pyqlib-0.9.3.99-py3.10-win-amd64.egg\\qlib\\data\\data.py:912\u001B[0m, in \u001B[0;36mLocalDatasetProvider.dataset\u001B[1;34m(self, instruments, fields, start_time, end_time, freq, inst_processors)\u001B[0m\n\u001B[0;32m    903\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mdataset\u001B[39m(\n\u001B[0;32m    904\u001B[0m     \u001B[38;5;28mself\u001B[39m,\n\u001B[0;32m    905\u001B[0m     instruments,\n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m    910\u001B[0m     inst_processors\u001B[38;5;241m=\u001B[39m[],\n\u001B[0;32m    911\u001B[0m ):\n\u001B[1;32m--> 912\u001B[0m     instruments_d \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mget_instruments_d\u001B[49m\u001B[43m(\u001B[49m\u001B[43minstruments\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfreq\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    913\u001B[0m     column_names \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mget_column_names(fields)\n\u001B[0;32m    914\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39malign_time:\n\u001B[0;32m    915\u001B[0m         \u001B[38;5;66;03m# NOTE: if the frequency is a fixed value.\u001B[39;00m\n\u001B[0;32m    916\u001B[0m         \u001B[38;5;66;03m# align the data to fixed calendar point\u001B[39;00m\n",
      "File \u001B[1;32mE:\\anaconda3\\lib\\site-packages\\pyqlib-0.9.3.99-py3.10-win-amd64.egg\\qlib\\data\\data.py:520\u001B[0m, in \u001B[0;36mDatasetProvider.get_instruments_d\u001B[1;34m(instruments, freq)\u001B[0m\n\u001B[0;32m    517\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(instruments, \u001B[38;5;28mdict\u001B[39m):\n\u001B[0;32m    518\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mmarket\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;129;01min\u001B[39;00m instruments:\n\u001B[0;32m    519\u001B[0m         \u001B[38;5;66;03m# dict of stockpool config\u001B[39;00m\n\u001B[1;32m--> 520\u001B[0m         instruments_d \u001B[38;5;241m=\u001B[39m \u001B[43mInst\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mlist_instruments\u001B[49m\u001B[43m(\u001B[49m\u001B[43minstruments\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43minstruments\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfreq\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mfreq\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mas_list\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m)\u001B[49m\n\u001B[0;32m    521\u001B[0m     \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m    522\u001B[0m         \u001B[38;5;66;03m# dict of instruments and timestamp\u001B[39;00m\n\u001B[0;32m    523\u001B[0m         instruments_d \u001B[38;5;241m=\u001B[39m instruments\n",
      "File \u001B[1;32mE:\\anaconda3\\lib\\site-packages\\pyqlib-0.9.3.99-py3.10-win-amd64.egg\\qlib\\data\\data.py:697\u001B[0m, in \u001B[0;36mLocalInstrumentProvider.list_instruments\u001B[1;34m(self, instruments, start_time, end_time, freq, as_list)\u001B[0m\n\u001B[0;32m    695\u001B[0m     _instruments \u001B[38;5;241m=\u001B[39m H[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mi\u001B[39m\u001B[38;5;124m\"\u001B[39m][market]\n\u001B[0;32m    696\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m--> 697\u001B[0m     _instruments \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_load_instruments\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmarket\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfreq\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mfreq\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    698\u001B[0m     H[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mi\u001B[39m\u001B[38;5;124m\"\u001B[39m][market] \u001B[38;5;241m=\u001B[39m _instruments\n\u001B[0;32m    699\u001B[0m \u001B[38;5;66;03m# strip\u001B[39;00m\n\u001B[0;32m    700\u001B[0m \u001B[38;5;66;03m# use calendar boundary\u001B[39;00m\n",
      "File \u001B[1;32mE:\\anaconda3\\lib\\site-packages\\pyqlib-0.9.3.99-py3.10-win-amd64.egg\\qlib\\data\\data.py:690\u001B[0m, in \u001B[0;36mLocalInstrumentProvider._load_instruments\u001B[1;34m(self, market, freq)\u001B[0m\n\u001B[0;32m    689\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m_load_instruments\u001B[39m(\u001B[38;5;28mself\u001B[39m, market, freq):\n\u001B[1;32m--> 690\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mbackend_obj\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmarket\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mmarket\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfreq\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mfreq\u001B[49m\u001B[43m)\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdata\u001B[49m\n",
      "File \u001B[1;32mE:\\anaconda3\\lib\\site-packages\\pyqlib-0.9.3.99-py3.10-win-amd64.egg\\qlib\\data\\storage\\file_storage.py:243\u001B[0m, in \u001B[0;36mFileInstrumentStorage.data\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m    241\u001B[0m \u001B[38;5;129m@property\u001B[39m\n\u001B[0;32m    242\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mdata\u001B[39m(\u001B[38;5;28mself\u001B[39m) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m Dict[InstKT, InstVT]:\n\u001B[1;32m--> 243\u001B[0m     \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcheck\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    244\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_read_instrument()\n",
      "File \u001B[1;32mE:\\anaconda3\\lib\\site-packages\\pyqlib-0.9.3.99-py3.10-win-amd64.egg\\qlib\\data\\storage\\file_storage.py:72\u001B[0m, in \u001B[0;36mFileStorageMixin.check\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m     65\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mcheck\u001B[39m(\u001B[38;5;28mself\u001B[39m):\n\u001B[0;32m     66\u001B[0m \u001B[38;5;250m    \u001B[39m\u001B[38;5;124;03m\"\"\"check self.uri\u001B[39;00m\n\u001B[0;32m     67\u001B[0m \n\u001B[0;32m     68\u001B[0m \u001B[38;5;124;03m    Raises\u001B[39;00m\n\u001B[0;32m     69\u001B[0m \u001B[38;5;124;03m    -------\u001B[39;00m\n\u001B[0;32m     70\u001B[0m \u001B[38;5;124;03m    ValueError\u001B[39;00m\n\u001B[0;32m     71\u001B[0m \u001B[38;5;124;03m    \"\"\"\u001B[39;00m\n\u001B[1;32m---> 72\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43muri\u001B[49m\u001B[38;5;241m.\u001B[39mexists():\n\u001B[0;32m     73\u001B[0m         \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;132;01m{\u001B[39;00m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mstorage_name\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m not exists: \u001B[39m\u001B[38;5;132;01m{\u001B[39;00m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39muri\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m\"\u001B[39m)\n",
      "File \u001B[1;32mE:\\anaconda3\\lib\\site-packages\\pyqlib-0.9.3.99-py3.10-win-amd64.egg\\qlib\\data\\storage\\file_storage.py:62\u001B[0m, in \u001B[0;36mFileStorageMixin.uri\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m     59\u001B[0m \u001B[38;5;129m@property\u001B[39m\n\u001B[0;32m     60\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21muri\u001B[39m(\u001B[38;5;28mself\u001B[39m) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m Path:\n\u001B[0;32m     61\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mfreq \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39msupport_freq:\n\u001B[1;32m---> 62\u001B[0m         \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;132;01m{\u001B[39;00m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mstorage_name\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m: \u001B[39m\u001B[38;5;132;01m{\u001B[39;00m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mprovider_uri\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m does not contain data for \u001B[39m\u001B[38;5;132;01m{\u001B[39;00m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mfreq\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m\"\u001B[39m)\n\u001B[0;32m     63\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mdpm\u001B[38;5;241m.\u001B[39mget_data_uri(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mfreq)\u001B[38;5;241m.\u001B[39mjoinpath(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;132;01m{\u001B[39;00m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mstorage_name\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124ms\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mfile_name)\n",
      "\u001B[1;31mValueError\u001B[0m: instrument: {'__DEFAULT_FREQ': 'C:\\\\Users\\\\胡逸凡\\\\.qlib\\\\qlib_data\\\\cn_data'} does not contain data for day"
     ]
    }
   ],
   "source": [
    "h = Alpha158(**data_handler_config)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-29T03:19:51.493733Z",
     "start_time": "2024-02-29T03:19:51.288961Z"
    }
   },
   "execution_count": 10
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "data_df = h.fetch()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-29T03:06:30.934782Z",
     "start_time": "2024-02-29T03:06:30.933782Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_df = data_df.reset_index().dropna(axis=1, how=\"all\").rename(\n",
    "    columns={\"datetime\": \"date\", \"instrument\": \"tic\"}\n",
    ")\n",
    "feature_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-02-29T03:06:30.939815Z"
    }
   },
   "outputs": [],
   "source": [
    "# Strange features that have different NaN data among different stocks.\n",
    "\n",
    "drop_col = []\n",
    "for k,v in feature_df.isna().sum().items():\n",
    "    if v % 100 != 0:\n",
    "        print(k, v)\n",
    "        drop_col.append(k)\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_df = feature_df.drop(columns=drop_col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-02-29T03:06:30.943817Z"
    }
   },
   "outputs": [],
   "source": [
    "feature_df = feature_df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-02-29T03:06:30.944817Z"
    }
   },
   "outputs": [],
   "source": [
    "for tic in feature_df['tic'].unique():\n",
    "    print(len(feature_df[feature_df['tic'] == tic]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-01T03:30:48.254310Z",
     "start_time": "2024-03-01T03:30:48.232013Z"
    }
   },
   "outputs": [],
   "source": [
    "# alpha158 = feature_df.columns[2:].to_list() # alpha158\n",
    "basic_feature = [\"open\", \"close\", \"high\", \"low\", \"volume\"]"
   ]
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "target_return_span = 5\n",
    "target = f\"return+{target_return_span}\""
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-01T03:30:52.329576Z",
     "start_time": "2024-03-01T03:30:52.315963Z"
    }
   },
   "execution_count": 19
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-01T03:30:54.482612Z",
     "start_time": "2024-03-01T03:30:54.468368Z"
    }
   },
   "outputs": [],
   "source": [
    "time_span = 60\n",
    "\n",
    "CSI_date = ['20110419', '20181228', '20180102', '20201231',  '20190402', '20211231']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-01T03:30:58.628381Z",
     "start_time": "2024-03-01T03:30:56.127476Z"
    }
   },
   "outputs": [],
   "source": [
    "tic_df_list = []\n",
    "for tic_path in os.listdir(\"./CSI/\"):\n",
    "    tic_df = pd.read_csv(f\"./CSI/{tic_path}\", index_col=0)[[\"date\", \"open\", \"close\", \"high\", \"low\", \"volume\"]]\n",
    "    tic_df[\"tic\"] = tic_path[:8]\n",
    "    tic_df[target] = tic_df.close.pct_change(target_return_span).shift(-1 * target_return_span)\n",
    "    tic_df_list.append(tic_df)\n",
    "\n",
    "tic_target_df = pd.concat(tic_df_list).sort_values(by=\"date\").dropna()\n",
    "tic_target_df.date = pd.DatetimeIndex(tic_target_df.date)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-01T03:31:01.052861Z",
     "start_time": "2024-03-01T03:31:00.993278Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "           date       open      close       high        low       volume  \\\n0    2010-01-04   9.032978   5.803878   9.049530   8.755741  159964606.0   \n0    2010-01-04  16.675959  10.920465  16.843369  16.424843   80224538.0   \n0    2010-01-04  17.716665  11.288666  18.693333  17.716665   11110185.0   \n0    2010-01-04  31.799999  25.086197  32.000000  31.360001   26409039.0   \n0    2010-01-04   7.370151   4.855292   7.422748   7.192636   84411995.0   \n...         ...        ...        ...        ...        ...          ...   \n2989 2022-04-26  22.920000  23.190001  23.389999  22.860001   43810129.0   \n2989 2022-04-26   6.880000   6.820000   6.990000   6.800000   46892044.0   \n2989 2022-04-26  34.200001  35.709999  36.500000  33.980000   74157971.0   \n2989 2022-04-26  32.349998  32.880001  34.049999  31.900000   23586836.0   \n2989 2022-04-26  14.880000  14.730000  14.980000  14.450000   98820956.0   \n\n           tic  return+5  \n0     SH600000 -0.023596  \n0     SH600036 -0.043478  \n0     SZ000895 -0.047533  \n0     SZ000858 -0.056916  \n0     SH600048 -0.041096  \n...        ...       ...  \n2989  SH600900 -0.020267  \n2989  SH601006 -0.046921  \n2989  SZ300015  0.013162  \n2989  SH600660  0.095195  \n2989  SZ000001  0.015614  \n\n[299000 rows x 8 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>date</th>\n      <th>open</th>\n      <th>close</th>\n      <th>high</th>\n      <th>low</th>\n      <th>volume</th>\n      <th>tic</th>\n      <th>return+5</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>2010-01-04</td>\n      <td>9.032978</td>\n      <td>5.803878</td>\n      <td>9.049530</td>\n      <td>8.755741</td>\n      <td>159964606.0</td>\n      <td>SH600000</td>\n      <td>-0.023596</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>2010-01-04</td>\n      <td>16.675959</td>\n      <td>10.920465</td>\n      <td>16.843369</td>\n      <td>16.424843</td>\n      <td>80224538.0</td>\n      <td>SH600036</td>\n      <td>-0.043478</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>2010-01-04</td>\n      <td>17.716665</td>\n      <td>11.288666</td>\n      <td>18.693333</td>\n      <td>17.716665</td>\n      <td>11110185.0</td>\n      <td>SZ000895</td>\n      <td>-0.047533</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>2010-01-04</td>\n      <td>31.799999</td>\n      <td>25.086197</td>\n      <td>32.000000</td>\n      <td>31.360001</td>\n      <td>26409039.0</td>\n      <td>SZ000858</td>\n      <td>-0.056916</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>2010-01-04</td>\n      <td>7.370151</td>\n      <td>4.855292</td>\n      <td>7.422748</td>\n      <td>7.192636</td>\n      <td>84411995.0</td>\n      <td>SH600048</td>\n      <td>-0.041096</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2989</th>\n      <td>2022-04-26</td>\n      <td>22.920000</td>\n      <td>23.190001</td>\n      <td>23.389999</td>\n      <td>22.860001</td>\n      <td>43810129.0</td>\n      <td>SH600900</td>\n      <td>-0.020267</td>\n    </tr>\n    <tr>\n      <th>2989</th>\n      <td>2022-04-26</td>\n      <td>6.880000</td>\n      <td>6.820000</td>\n      <td>6.990000</td>\n      <td>6.800000</td>\n      <td>46892044.0</td>\n      <td>SH601006</td>\n      <td>-0.046921</td>\n    </tr>\n    <tr>\n      <th>2989</th>\n      <td>2022-04-26</td>\n      <td>34.200001</td>\n      <td>35.709999</td>\n      <td>36.500000</td>\n      <td>33.980000</td>\n      <td>74157971.0</td>\n      <td>SZ300015</td>\n      <td>0.013162</td>\n    </tr>\n    <tr>\n      <th>2989</th>\n      <td>2022-04-26</td>\n      <td>32.349998</td>\n      <td>32.880001</td>\n      <td>34.049999</td>\n      <td>31.900000</td>\n      <td>23586836.0</td>\n      <td>SH600660</td>\n      <td>0.095195</td>\n    </tr>\n    <tr>\n      <th>2989</th>\n      <td>2022-04-26</td>\n      <td>14.880000</td>\n      <td>14.730000</td>\n      <td>14.980000</td>\n      <td>14.450000</td>\n      <td>98820956.0</td>\n      <td>SZ000001</td>\n      <td>0.015614</td>\n    </tr>\n  </tbody>\n</table>\n<p>299000 rows × 8 columns</p>\n</div>"
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tic_target_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-01T03:31:13.804715Z",
     "start_time": "2024-03-01T03:31:13.775298Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "           date       open      close       high        low       volume  \\\n0    2010-01-04   9.032978   5.803878   9.049530   8.755741  159964606.0   \n0    2010-01-04  16.675959  10.920465  16.843369  16.424843   80224538.0   \n0    2010-01-04  17.716665  11.288666  18.693333  17.716665   11110185.0   \n0    2010-01-04  31.799999  25.086197  32.000000  31.360001   26409039.0   \n0    2010-01-04   7.370151   4.855292   7.422748   7.192636   84411995.0   \n...         ...        ...        ...        ...        ...          ...   \n2989 2022-04-26  22.920000  23.190001  23.389999  22.860001   43810129.0   \n2989 2022-04-26   6.880000   6.820000   6.990000   6.800000   46892044.0   \n2989 2022-04-26  34.200001  35.709999  36.500000  33.980000   74157971.0   \n2989 2022-04-26  32.349998  32.880001  34.049999  31.900000   23586836.0   \n2989 2022-04-26  14.880000  14.730000  14.980000  14.450000   98820956.0   \n\n           tic  return+5  \n0     SH600000 -0.023596  \n0     SH600036 -0.043478  \n0     SZ000895 -0.047533  \n0     SZ000858 -0.056916  \n0     SH600048 -0.041096  \n...        ...       ...  \n2989  SH600900 -0.020267  \n2989  SH601006 -0.046921  \n2989  SZ300015  0.013162  \n2989  SH600660  0.095195  \n2989  SZ000001  0.015614  \n\n[299000 rows x 8 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>date</th>\n      <th>open</th>\n      <th>close</th>\n      <th>high</th>\n      <th>low</th>\n      <th>volume</th>\n      <th>tic</th>\n      <th>return+5</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>2010-01-04</td>\n      <td>9.032978</td>\n      <td>5.803878</td>\n      <td>9.049530</td>\n      <td>8.755741</td>\n      <td>159964606.0</td>\n      <td>SH600000</td>\n      <td>-0.023596</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>2010-01-04</td>\n      <td>16.675959</td>\n      <td>10.920465</td>\n      <td>16.843369</td>\n      <td>16.424843</td>\n      <td>80224538.0</td>\n      <td>SH600036</td>\n      <td>-0.043478</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>2010-01-04</td>\n      <td>17.716665</td>\n      <td>11.288666</td>\n      <td>18.693333</td>\n      <td>17.716665</td>\n      <td>11110185.0</td>\n      <td>SZ000895</td>\n      <td>-0.047533</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>2010-01-04</td>\n      <td>31.799999</td>\n      <td>25.086197</td>\n      <td>32.000000</td>\n      <td>31.360001</td>\n      <td>26409039.0</td>\n      <td>SZ000858</td>\n      <td>-0.056916</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>2010-01-04</td>\n      <td>7.370151</td>\n      <td>4.855292</td>\n      <td>7.422748</td>\n      <td>7.192636</td>\n      <td>84411995.0</td>\n      <td>SH600048</td>\n      <td>-0.041096</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2989</th>\n      <td>2022-04-26</td>\n      <td>22.920000</td>\n      <td>23.190001</td>\n      <td>23.389999</td>\n      <td>22.860001</td>\n      <td>43810129.0</td>\n      <td>SH600900</td>\n      <td>-0.020267</td>\n    </tr>\n    <tr>\n      <th>2989</th>\n      <td>2022-04-26</td>\n      <td>6.880000</td>\n      <td>6.820000</td>\n      <td>6.990000</td>\n      <td>6.800000</td>\n      <td>46892044.0</td>\n      <td>SH601006</td>\n      <td>-0.046921</td>\n    </tr>\n    <tr>\n      <th>2989</th>\n      <td>2022-04-26</td>\n      <td>34.200001</td>\n      <td>35.709999</td>\n      <td>36.500000</td>\n      <td>33.980000</td>\n      <td>74157971.0</td>\n      <td>SZ300015</td>\n      <td>0.013162</td>\n    </tr>\n    <tr>\n      <th>2989</th>\n      <td>2022-04-26</td>\n      <td>32.349998</td>\n      <td>32.880001</td>\n      <td>34.049999</td>\n      <td>31.900000</td>\n      <td>23586836.0</td>\n      <td>SH600660</td>\n      <td>0.095195</td>\n    </tr>\n    <tr>\n      <th>2989</th>\n      <td>2022-04-26</td>\n      <td>14.880000</td>\n      <td>14.730000</td>\n      <td>14.980000</td>\n      <td>14.450000</td>\n      <td>98820956.0</td>\n      <td>SZ000001</td>\n      <td>0.015614</td>\n    </tr>\n  </tbody>\n</table>\n<p>299000 rows × 8 columns</p>\n</div>"
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dataset_df = feature_df.merge(tic_target_df, how='inner', on=[\"date\", \"tic\"])\n",
    "dataset_df = tic_target_df\n",
    "dataset_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-01T03:31:14.878973Z",
     "start_time": "2024-03-01T03:31:14.834440Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 299000 entries, 0 to 2989\n",
      "Data columns (total 8 columns):\n",
      " #   Column    Non-Null Count   Dtype         \n",
      "---  ------    --------------   -----         \n",
      " 0   date      299000 non-null  datetime64[ns]\n",
      " 1   open      299000 non-null  float64       \n",
      " 2   close     299000 non-null  float64       \n",
      " 3   high      299000 non-null  float64       \n",
      " 4   low       299000 non-null  float64       \n",
      " 5   volume    299000 non-null  float64       \n",
      " 6   tic       299000 non-null  object        \n",
      " 7   return+5  299000 non-null  float64       \n",
      "dtypes: datetime64[ns](1), float64(6), object(1)\n",
      "memory usage: 20.5+ MB\n"
     ]
    }
   ],
   "source": [
    "dataset_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "2990"
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_df['date'].nunique()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-01T05:51:28.325562Z",
     "start_time": "2024-03-01T05:51:28.259706Z"
    }
   },
   "execution_count": 35
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-01T03:31:17.547819Z",
     "start_time": "2024-03-01T03:31:17.500813Z"
    }
   },
   "outputs": [],
   "source": [
    "train_df = dataset_df[(dataset_df.date >= CSI_date[0]) & (dataset_df.date <= CSI_date[1])]\n",
    "val_df = dataset_df[(dataset_df.date >= CSI_date[2]) & (dataset_df.date <= CSI_date[3])]\n",
    "test_df = dataset_df[(dataset_df.date >= CSI_date[4]) & (dataset_df.date <= CSI_date[5])]"
   ]
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "669   1877   728\n"
     ]
    }
   ],
   "source": [
    "print(test_df['date'].nunique(), ' ', train_df['date'].nunique(), ' ', val_df['date'].nunique())"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-01T05:52:57.873322Z",
     "start_time": "2024-03-01T05:52:57.845744Z"
    }
   },
   "execution_count": 38
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-01T03:31:19.346383Z",
     "start_time": "2024-03-01T03:31:19.338382Z"
    }
   },
   "outputs": [],
   "source": [
    "# tic_list = dataset_df.tic.unique()\n",
    "\n",
    "# with open(\"tic_list.txt\", \"w\") as f:\n",
    "#     for tic in tic_list:\n",
    "#         f.write(tic + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-01T03:31:22.150061Z",
     "start_time": "2024-03-01T03:31:22.136547Z"
    }
   },
   "outputs": [],
   "source": [
    "with open(\"tic_list.txt\", \"r\") as f:\n",
    "    tic_list = [tic.strip(\"\\n\") for tic in f.readlines()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-01T03:31:23.430981Z",
     "start_time": "2024-03-01T03:31:23.412922Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-01T03:32:41.270835Z",
     "start_time": "2024-03-01T03:32:41.245270Z"
    }
   },
   "outputs": [],
   "source": [
    "def df_2_array(dataset_df, feat_col, target, type):\n",
    "    dataset_feat = []\n",
    "    dataset_ret = []\n",
    "    dataset_price = []\n",
    "    for tic in tic_list:\n",
    "        df = dataset_df[dataset_df.tic == tic]\n",
    "        feat = df[feat_col].to_numpy()\n",
    "        ret = df[target].to_numpy()\n",
    "        price = df['close'].to_numpy()\n",
    "        stock_feat = []\n",
    "        stock_ret = []\n",
    "        stock_price = []\n",
    "        for i in range(time_span, feat.shape[0]):\n",
    "            stock_feat.append(feat[i-time_span : i])\n",
    "            stock_ret.append(ret[i])\n",
    "            stock_price.append(price[i])\n",
    "        stock_feat = np.array(stock_feat)\n",
    "        stock_ret = np.array(stock_ret)\n",
    "        stock_price = np.array(stock_price)\n",
    "    \n",
    "        dataset_feat.append(stock_feat)\n",
    "        dataset_ret.append(stock_ret)\n",
    "        dataset_price.append(stock_price)\n",
    "\n",
    "    dataset_feat = np.array(dataset_feat).transpose((1, 2, 0, 3))\n",
    "    dataset_ret = np.array(dataset_ret).transpose((1, 0))\n",
    "    dataset_price = np.array(dataset_price).transpose((1, 0))\n",
    "    \n",
    "    dataset_feat_tensor = torch.tensor(dataset_feat, dtype=torch.float)\n",
    "    dataset_ret_tensor = torch.tensor(dataset_ret, dtype=torch.float)\n",
    "    dataset_price_tensor = torch.tensor(dataset_price, dtype=torch.float)\n",
    "    \n",
    "    torch.save(dataset_feat_tensor, f\"./dataset/{type}/feat.pt\")\n",
    "    torch.save(dataset_ret_tensor, f\"./dataset/{type}/ret.pt\")\n",
    "    torch.save(dataset_price_tensor, f\"./dataset/{type}/price.pt\")    \n",
    "    \n",
    "    return dataset_feat, dataset_ret, dataset_price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-01T03:32:47.170989Z",
     "start_time": "2024-03-01T03:32:41.735067Z"
    }
   },
   "outputs": [],
   "source": [
    "_ = df_2_array(train_df, basic_feature, target, \"train\")\n",
    "_ = df_2_array(val_df, basic_feature, target, \"val\")\n",
    "dataset_feat, dataset_ret, dataset_price = df_2_array(test_df, basic_feature, target, \"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-01T03:32:51.632666Z",
     "start_time": "2024-03-01T03:32:51.617141Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "(609, 60, 100, 5)"
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_feat.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-01T03:32:52.537761Z",
     "start_time": "2024-03-01T03:32:52.526662Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "(609, 100)"
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_ret.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.13 ('finrl')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "fdc32ee0fbebcc1c7f930cbcb8cecd692d57289f6e3d9429e6f0d4e8176a8c57"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
